{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from category_encoders import OrdinalEncoder\n",
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from bayes_opt import bayesian_optimization\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train dataset shape is: (9674, 15)\n",
      "test dataset shape is: (11182, 15)\n"
     ]
    }
   ],
   "source": [
    "# question 1: read adult data.txt as train, adult test.txt as test, rename their headers to ['age', \n",
    "# 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', 'occupation', 'relationship',\n",
    "# 'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country', 'wage_class']\n",
    "column_name = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', \n",
    "               'occupation', 'relationship', 'race', 'sex', 'capital_gain', 'capital_loss', \n",
    "               'hours_per_week', 'native_country', 'wage_class']\n",
    "df_train = pd.read_csv('adult+data.txt', header = None)\n",
    "df_test = pd.read_csv('adult+test.txt', header = None)\n",
    "df_train.columns = column_name\n",
    "df_test.columns = column_name\n",
    "print(\"train dataset shape is: \" + str(df_train.shape))\n",
    "print(\"test dataset shape is: \" + str(df_test.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train dataset shape without '?' is: (8936, 15)\n",
      "test dataset shape without '?' is: (10347, 15)\n"
     ]
    }
   ],
   "source": [
    "# question 2: replace ‘ ?’ with nan and drop na\n",
    "df_train_nona = df_train.replace(' ?', np.nan).dropna()\n",
    "df_test_nona = df_test.replace(' ?', np.nan).dropna()\n",
    "print(\"train dataset shape without '?' is: \" + str(df_train_nona.shape))\n",
    "print(\"test dataset shape without '?' is: \" + str(df_test_nona.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# question 3: replace test wage_class all of the ‘<=50K.’ \n",
    "# with ‘<=50K’ and the same for ‘>50K.’ with ‘>50K’, same as train\n",
    "df_test_rep = df_test_nona.replace([' >50K.', ' <=50K.'], ['>50K', '<=50K'])\n",
    "df_train_rep = df_train_nona.replace([' >50K', ' <=50K'], ['>50K', '<=50K'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education</th>\n",
       "      <th>education_num</th>\n",
       "      <th>marital_status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>capital_gain</th>\n",
       "      <th>capital_loss</th>\n",
       "      <th>hours_per_week</th>\n",
       "      <th>native_country</th>\n",
       "      <th>wage_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>77516</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2174</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50</td>\n",
       "      <td>2</td>\n",
       "      <td>83311</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38</td>\n",
       "      <td>3</td>\n",
       "      <td>215646</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53</td>\n",
       "      <td>3</td>\n",
       "      <td>234721</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>338409</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  workclass  fnlwgt  education  education_num  marital_status  \\\n",
       "0   39          1   77516          1             13               1   \n",
       "1   50          2   83311          1             13               2   \n",
       "2   38          3  215646          2              9               3   \n",
       "3   53          3  234721          3              7               2   \n",
       "4   28          3  338409          1             13               2   \n",
       "\n",
       "   occupation  relationship  race  sex  capital_gain  capital_loss  \\\n",
       "0           1             1     1    1          2174             0   \n",
       "1           2             2     1    1             0             0   \n",
       "2           3             1     1    1             0             0   \n",
       "3           3             2     2    1             0             0   \n",
       "4           4             3     2    2             0             0   \n",
       "\n",
       "   hours_per_week  native_country  wage_class  \n",
       "0              40               1           0  \n",
       "1              13               1           0  \n",
       "2              40               1           0  \n",
       "3              40               1           0  \n",
       "4              40               2           0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# question 4: apply Ordinal Encoding to Categoricals for both train and test\n",
    "columns_object = df_train_rep.select_dtypes(include = 'object').columns\n",
    "df_train_final = df_train_rep\n",
    "df_test_final = df_test_rep\n",
    "for i in df_train_rep.columns:\n",
    "    if df_train_rep[i].dtype == 'object':\n",
    "        encoder = OrdinalEncoder()\n",
    "        df_train_final[i] = encoder.fit_transform(df_train_rep[i])\n",
    "for i in df_test_rep.columns:\n",
    "    if df_test_rep[i].dtype == 'object':\n",
    "        encoder = OrdinalEncoder()\n",
    "        df_test_final[i] = encoder.fit_transform(df_test_rep[i])\n",
    "df_train_final['wage_class'] = df_train_final['wage_class'] - 1\n",
    "df_test_final['wage_class'] = df_test_final['wage_class'] - 1\n",
    "df_train_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_final['wage_class'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = df_train_final['wage_class']\n",
    "y_test = df_test_final['wage_class']\n",
    "x_train = df_train_final.drop(columns = 'wage_class')\n",
    "x_test = df_test_final.drop(columns = 'wage_class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... |   gamma   | learni... | max_depth | min_ch... | n_esti... | subsample |\n",
      "-------------------------------------------------------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.918   \u001b[0m | \u001b[0m 0.8567  \u001b[0m | \u001b[0m 0.8171  \u001b[0m | \u001b[0m 0.01147 \u001b[0m | \u001b[0m 4.718   \u001b[0m | \u001b[0m 3.442   \u001b[0m | \u001b[0m 1.988e+0\u001b[0m | \u001b[0m 0.7029  \u001b[0m |\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m 0.9089  \u001b[0m | \u001b[0m 0.7522  \u001b[0m | \u001b[0m 0.09978 \u001b[0m | \u001b[0m 0.03409 \u001b[0m | \u001b[0m 6.836   \u001b[0m | \u001b[0m 3.584   \u001b[0m | \u001b[0m 1.66e+03\u001b[0m | \u001b[0m 0.8015  \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 0.9052  \u001b[0m | \u001b[0m 0.7296  \u001b[0m | \u001b[0m 0.8534  \u001b[0m | \u001b[0m 0.09629 \u001b[0m | \u001b[0m 6.936   \u001b[0m | \u001b[0m 2.135   \u001b[0m | \u001b[0m 2e+03   \u001b[0m | \u001b[0m 0.8739  \u001b[0m |\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 0.9118  \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 1e+03   \u001b[0m | \u001b[0m 0.7     \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.9135  \u001b[0m | \u001b[0m 0.8888  \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.01    \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 1.274e+0\u001b[0m | \u001b[0m 0.7     \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 0.9087  \u001b[0m | \u001b[0m 0.8469  \u001b[0m | \u001b[0m 0.7904  \u001b[0m | \u001b[0m 0.0363  \u001b[0m | \u001b[0m 6.82    \u001b[0m | \u001b[0m 1.066   \u001b[0m | \u001b[0m 1.839e+0\u001b[0m | \u001b[0m 0.8013  \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 0.9145  \u001b[0m | \u001b[0m 0.7982  \u001b[0m | \u001b[0m 0.5256  \u001b[0m | \u001b[0m 0.05028 \u001b[0m | \u001b[0m 3.055   \u001b[0m | \u001b[0m 4.92    \u001b[0m | \u001b[0m 1.459e+0\u001b[0m | \u001b[0m 0.7279  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.9126  \u001b[0m | \u001b[0m 0.8981  \u001b[0m | \u001b[0m 0.4358  \u001b[0m | \u001b[0m 0.09928 \u001b[0m | \u001b[0m 3.031   \u001b[0m | \u001b[0m 4.908   \u001b[0m | \u001b[0m 1.122e+0\u001b[0m | \u001b[0m 0.8487  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.9088  \u001b[0m | \u001b[0m 0.761   \u001b[0m | \u001b[0m 0.1372  \u001b[0m | \u001b[0m 0.09364 \u001b[0m | \u001b[0m 3.051   \u001b[0m | \u001b[0m 4.867   \u001b[0m | \u001b[0m 1.913e+0\u001b[0m | \u001b[0m 0.7348  \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.9148  \u001b[0m | \u001b[0m 0.7795  \u001b[0m | \u001b[0m 0.1265  \u001b[0m | \u001b[0m 0.04745 \u001b[0m | \u001b[0m 3.009   \u001b[0m | \u001b[0m 4.832   \u001b[0m | \u001b[0m 1.752e+0\u001b[0m | \u001b[0m 0.8045  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.911   \u001b[0m | \u001b[0m 0.8717  \u001b[0m | \u001b[0m 0.6882  \u001b[0m | \u001b[0m 0.08229 \u001b[0m | \u001b[0m 3.136   \u001b[0m | \u001b[0m 1.015   \u001b[0m | \u001b[0m 1.555e+0\u001b[0m | \u001b[0m 0.8436  \u001b[0m |\n",
      "| \u001b[95m 12      \u001b[0m | \u001b[95m 0.9188  \u001b[0m | \u001b[95m 0.7     \u001b[0m | \u001b[95m 0.1139  \u001b[0m | \u001b[95m 0.01    \u001b[0m | \u001b[95m 7.0     \u001b[0m | \u001b[95m 1.0     \u001b[0m | \u001b[95m 1.365e+0\u001b[0m | \u001b[95m 0.7     \u001b[0m |\n",
      "=============================================================================================================\n",
      "-----------------------------------------------------\n",
      "Final Results\n",
      "XGBOOST: 0.918824\n"
     ]
    }
   ],
   "source": [
    "# question 5: build xgboost: wage_class is the response variable, and others are predictors\n",
    "# use bayesian to find the best combination of tunning parameters:\n",
    "# 'max_depth': (3,7) 'min_child_weight': (1,5), 'learning_rate': (0.01, 0.1), 'subsample': (0.7,0.9)\n",
    "# 'colsample_bytree': (0.7,0.9), 'n_estimators': (1000, 2000), ' 'gamma': (0.01,1.)\n",
    "def xgboostcv(max_depth, learning_rate, n_estimators, gamma, min_child_weight, \n",
    "              subsample, colsample_bytree, silent = True, nthread = -1):\n",
    "    return cross_val_score(xgb.XGBClassifier(max_depth = int(max_depth), learning_rate = learning_rate, \n",
    "                                             n_estimators = int(n_estimators), silent = silent, nthread = nthread, \n",
    "                                             gamma = gamma, min_child_weight = min_child_weight, subsample = subsample, \n",
    "                                             colsample_bytree = colsample_bytree),\n",
    "                           x_train, \n",
    "                           y_train, \n",
    "                           scoring = 'roc_auc', \n",
    "                           cv=5).mean()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    \n",
    "    xgboostBO = bayesian_optimization.BayesianOptimization(xgboostcv,\n",
    "                                 {'max_depth': (3, 7),\n",
    "                                  'learning_rate': (0.01, 0.1),\n",
    "                                  'n_estimators': (1000, 2000),\n",
    "                                  'gamma': (0.01, 1.),\n",
    "                                  'min_child_weight': (1, 5),\n",
    "                                  'subsample': (0.7, 0.9),\n",
    "                                  'colsample_bytree' :(0.7, 0.9)})\n",
    "    xgboostBO.maximize(init_points=2, n_iter = 10)\n",
    "    print('-'*53)\n",
    "    print('Final Results')\n",
    "    temp = xgboostBO.res\n",
    "    temp_list = []\n",
    "    for i in temp:\n",
    "        temp_list.append(i['target'])\n",
    "    print('XGBOOST: %f' % max(temp_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train-error-mean</th>\n",
       "      <th>train-error-std</th>\n",
       "      <th>test-error-mean</th>\n",
       "      <th>test-error-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.161034</td>\n",
       "      <td>0.002462</td>\n",
       "      <td>0.174463</td>\n",
       "      <td>0.013897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.148137</td>\n",
       "      <td>0.004938</td>\n",
       "      <td>0.159468</td>\n",
       "      <td>0.009311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.146766</td>\n",
       "      <td>0.002765</td>\n",
       "      <td>0.160586</td>\n",
       "      <td>0.008914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.141506</td>\n",
       "      <td>0.002025</td>\n",
       "      <td>0.155327</td>\n",
       "      <td>0.009329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.142961</td>\n",
       "      <td>0.001548</td>\n",
       "      <td>0.153872</td>\n",
       "      <td>0.010867</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   train-error-mean  train-error-std  test-error-mean  test-error-std\n",
       "0          0.161034         0.002462         0.174463        0.013897\n",
       "1          0.148137         0.004938         0.159468        0.009311\n",
       "2          0.146766         0.002765         0.160586        0.008914\n",
       "3          0.141506         0.002025         0.155327        0.009329\n",
       "4          0.142961         0.001548         0.153872        0.010867"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# question 6: use xgb.cv to find the early stopping rounds based on error\n",
    "train_initial = xgb.DMatrix(x_train, y_train)\n",
    "target_para = {}\n",
    "for i in temp:\n",
    "    if i['target'] == max(temp_list):\n",
    "        target_para = i\n",
    "param = {'eta': 0.01, 'seed':0, 'subsample': target_para['params']['subsample'], \n",
    "         'colsample_bytree': target_para['params']['colsample_bytree'], 'objective': 'binary:logistic', \n",
    "         'max_depth': int(target_para['params']['max_depth'].round()), \n",
    "         'min_child_weight': target_para['params']['min_child_weight'], \n",
    "         'gamma': target_para['params']['gamma']}\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "df_cv = xgb.cv(params = param, dtrain = train_initial, num_boost_round = 1000, \n",
    "               nfold = 5, metrics = ['error'], seed = 0, early_stopping_rounds = 100)\n",
    "df_cv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAEICAYAAAA3PAFIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de7hd073/8fenQYhLQqknlEZJaeKSksSdlFRLi7TVpi2t4NA6ru2PVksVPUrpaU9dWiccgioal3JoEZIQQSSRq7sSR0NdilTcxff3xxgrmVnZ12Ttvdae+/N6nv3sucaac8wx97Pla8w19/goIjAzM+vqPlLvAZiZmdWCC5qZmZWCC5qZmZWCC5qZmZWCC5qZmZWCC5qZmZWCC5pZg5H0sKRh9R6HWVcj/x2amTVF0jzg3yLiznqPxawtPEMzs6VIWqneYzBbHi5oZg1G0jxJwyWdJmmspD9IekPSHEmfkvRjSS9Jek7SXoXjJko6S9KDkv4l6SZJ6xTe3y/fznw97/vpqnP+SNJs4E1JVwMbA/8raaGkH+b9xkr6h6QFku6RNLDQxxhJF0q6NY93iqRNC+8PlDRO0quSXpT0k9z+EUknSfqbpH9K+lNx3GZt5YJm1tj2Ba4E1gZmALeT/rvdEDgD+O+q/b8DHAr0BT4AzgOQ9CngauB4YD3gL6RitUrh2G8CXwT6RMQ3gf8D9o2INSLinLzPX4H+wMeAh4Crqs7/DeD0PN6ngDPz+dcE7gRuAzYANgPuysccA4wAds/vvQZc2PYfkVnigmbW2CZFxO0R8QEwllSMzo6I94FrgH6S+hT2vzIi5kbEm8BPga9L6gGMBG6NiHH52F8BqwE7FY49LyKei4i3mxtMRFwaEW9ExLvAacA2knoXdrkxIh7M470KGJTbvwT8IyL+MyLeyX1Mye99Dzg5Iv5e6PcA3/q09nJBM2tsLxa23wZeiYhFhdcAaxT2ea6w/SywMrAuaebzbOWNiPgw77thM8cuQ1IPSWfnW4P/Aublt9Yt7PaPwvZbhbFtBPytma4/AdyYb4W+DjwKLALWb2k8ZtVc0MzKZaPC9sbA+8ArwPOkwgGAJOV95xf2r37kufr1t4D9geFAb6Bfpbs2jOs54JMtvLd3RPQpfK0aEfOb2d+sSS5oZuVykKQBknqRPmO7Ls/o/gR8UdKeklYG/h/wLnBfC329yNJFaM18zD+BXsAv2jGuW4C+ko6X1FPSmpK2z+9dBJwp6RMAktaTtH87+jYDXNDMyuZKYAzp1t+qwLEAEfE4cBBwPmnGti/pgY/3WujrLOCUfCvwBOAK0m3L+cAjwANtHVREvAF8Lp/3H8CTwGfz278FbgbukPRG7nf7pvoxa4n/sNqsJCRNBP4QEZfUeyxm9eAZmpmZlYILmpmZlYJvOZqZWSl4hmZmZqXgv8Svo3XXXTf69etX72GYmXUp06dPfyUi1qtud0Gro379+jFt2rR6D8PMrEuR9GxT7b7laGZmpeCCZmZmpeBbjnU0Z/4C+p10a72HYWbWqead/cUO6bfbzNAkHSvpUUnV+U3FfRbW4DyjJG2wov2YmVn7dKcZ2r8DwyPi7x18nlHAXNLq5mZm1km6xQxN0kWkVcP/mqPjL80R9E9LOraJ/S+UtF/evlHSpXn7UEmVBN6fSnpc0r2SrpZ0gqQDgMHAVZJmSlqt867SzKx76xYFLSK+R5oxfRb4DbAF8HlgKPCzHKdRNAnYNW9vCAzI27sC90gaAnwV2AbYm1TEiIjrgGnAgRExqKnkX0lHSJomadqitxbU8CrNzLq3blHQmnBrRLwbEa8AL7FsMu4kYFdJA0gxGS9K6gvsSMqP2hm4qRIlD/xvW08cEaMjYnBEDO7Rq3frB5iZWZt0p8/Qit4tbC+i6ucQEfMl9QG+ANwDrAN8HVgYEW+ksF8zM2sk3XWG1hYPAMeTCtok4IT8HWAysK+kVSWtAXypcNwbpGRfMzPrRN11htYWk4C9IuKpvMzKOrmNiJgq6WZgNimmfg5Q+UBsDHCRpLeBHZv6HK1iqw17M62D/h7DzKy7cXzMcpK0RkQslNSLNIs7IiIeak8fgwcPDq/laGbWPpKmR8Tg6nbP0Jbf6PzQyKrA5e0tZmZmVlsuaMspIr5V7zGYmdkSfijEzMxKwQXNzMxKwQXNzMxKwQXNzMxKwQ+F1JHz0My6lo7K8bLa8AzNzMxKwQWtBZL+LGm6pIclHZHbDpP0hKQHJV0s6YLcvp6k6yVNzV8713f0Zmbdi285tuzQiHg155pNlXQr8FNgW9KajeOBWXnf3wK/iYh7JW0M3A58uh6DNjPrjlzQWnaspC/n7Y2AbwN3R8SrAJLGAp/K7w8HBhRW4l+rsjxWscM80zsCoMda63Xw8M3Mug8XtGZIGkYqUjtGxFuSJgKP0fys6yPADhHxTkv9RsRoYDRAz779vZCmmVmN+DO05vUGXsvFbAtgB2B1YHdJa0taiZRaXXEHcEzlhaRBnTpaM7NuzgWtebcBK0l6FDiblI82H/gF8CApE20eS2JjjgUGS5ot6RHge50+YjOzbszxMe1UiI1ZCbgRuDQiblyevhwfY2bWfs3Fx3iG1n6nSZoJzAWeAf5c5/GYmRl+KKTdIuKEeo/BzMyW5RmamZmVgguamZmVgguamZmVgguamZmVgh8KqSPHx1g9OQrFysYzNDMzK4VWC5qkfpLmdsZgujL/nMzM6qsuM7S8ykbpzmVmZvXT1oLWI4dZPizpDkmrSRok6YG8duGNktYGkDRR0uC8va6keXl7lKSbJY0H7pLUV9I9kmZKmitp1+ZOLmmhpN/k898lab3cvqmk23II56S8iDCSxki6SNIU4Jxm+pwjqY+Sf0r6Tm6/QtLnJPWQdG4O65wt6buFY08stJ/eRN+flDRD0pAm3jtC0jRJ0xa9taD6bTMzW05tLWj9gQsjYiDwOmmV+SuAH0XE1sAc4Gdt6Gdb4ICI2B34FnB7RAwCtgFmtnDc6sC0fP67C+caDRwTEdsBJwC/KxzzcWCniPhBM31OBnYGBgJPA5WCuiNwH3AYsCAihgBDgMMlbSJpr/zzGAoMAraTtFulU0mbA9cDoyJiavVJI2J0RAyOiME9evVu4ZLNzKw92no77pmIqBSc6cCmQJ+IuDu3XQ6MbUM/4yrhmMBU4FJJKwN/LvTflA+Ba/P2H4AbJK0B7ASMLYRq9iwcMzYiFrXQ5yRgN+BZ4PfAEZI2JEXGvJkL19aSDsj79yYVsr3y14zcvkZu/z9gPeAm4CsR8UgL5zYzsxpr6wzt3cL2IqBPC/t+UOh31ar33qxsRMQ9pIIyHxhTueXXRpHP8XpEDCp8FcM332zm2Ip7SLOyXYGJwMvAAaRCByDS7K/S9yYRcUduP6vQvllE/E8+ZgGpsO3SjmsxM7MaWN4HJhYAr0naNSImAd8m3QqElBG2HSkz7ICmDwdJnwD+HhEXS+pJuh15RTO7fyT3dQ3pVuW9EfEvSc9I+lpEjFWapm0dEbPacgER8ZykdYFVIuJpSfeSblsenXe5HThS0viIeF/Sp0jF93bg55KuyjEyGwLv52PeA74M3C5pYUT8saUxbLVhb6b5b4HMzGpiRZ4APBi4SFIv0mdQh+T2XwF/knQE0NJfDQ8DTpT0PrAQaGmG9iYwVNIpwEvAyNx+IPD73L4yqeC1qaBlU4AeeXsScBZwb359CdAPeCgXy5eBERFxh6RPA/fnW50LgYNIM1fy7covAeNyUbu5HeMxM7Pl1CUCPnNhWKPe46g1B3yambWfAz7NzKzUGuqPjvPfjfWsav72iszOJB0CHFfVPDkijlrePs3MrPE0VEGLiO07oM/LgMtq3a+ZmTUW33I0M7NScEEzM7NSaKhbjt2N89CsHpyDZmXlGVoT8sLL+xRe7yfppHqOyczMWuaC1rRBwOKCFhE3R8TZdRyPmZm1oqEKmqQf5CiZuZKOz23fyTEtsyRdmdvWz5E1s/LXTtUBm5JOkHRa3p4o6beFqJqhuX2opPtz1Mt9kjaXtApwBjAy7z8yR99ckI/pJ2l8HtNdkjbO7WMknZf7ebqwqLGZmXWChvkMTdJ2pOWztictADxF0lTgFFIMzCuS1sm7nwfcHRFfltSDtOL92q2coldEDMpRL5cCWwKPAbtGxAeShgO/iIivSjoVGBwRR+exjSr0cz5weURcLunQPJYR+b2+pIWJtwBuBq5r4jqPAI4A6LHWem398ZiZWSsapqCRCsGNEfEmgKQbgMGkGJhXAArRM3uQ137METELlANGW3B13v8eSWtJ6gOsCVwuqT9pBf+V2zDOHYGv5O0rWTpA9M8R8SHwiKT1mzo4IkaTctzo2bd/4687ZmbWRTTULccVVIytgWWja6qLRwA/ByZExJbAvk0c017FmB01u5eZmdVcIxW0ScAISb0krU6KYZkGfE3SRwEKtxzvAo7MbT0k9QZeBD4m6aM5juZLVf2PzPvvQkqiXkAK7Zyf3x9V2PcN0uytKfcB38jbB7IkP83MzOqoYW45RsRDksaQctQALomIyZLOBO6WtIiUEj2KtDbjaEmHkWJbjoyI+yWdkY+fT/p8rOgdSTNItxUPzW3nkG45nsLSUTcTgJMkzSRFyhQdA1wm6URSpMwhLCfnoZmZ1U6XiI9ZUZImAidERENltTg+xsys/RwfY2ZmpdYwtxw7UkQMq/cYzMysY3mGZmZmpeCCZmZmpeCCZmZmpdAtPkNrVI6PKQfHsZg1Bs/QzMysFGpe0Ior09ewzxGSBhRen5EXEzYzMwO6zgxtBLC4oEXEqRFxZx3HY2ZmDabdBU3SQZIezFlh/53XUjxE0hOSHgR2Luw7ppgLJmlhYftHkubkPLOzc9vhkqbmtuvzuo47AfsB5+ZzblrsV9KeOc9sjqRL8zqOSJon6XRJD+X3tmjhmk7Lx07MWWbH5vbWMtZ+I2mapEclDZF0g6QnJf1HC+c6Ih8zbdFbC9r74zczs2a0q6BJ+jRpkd+dI2IQaR3Fg4DTSYVsFwozqRb62RvYH9g+IrZhSQTLDRExJLc9ChwWEfeRssVOjIhBEfG3Qj+rAmOAkRGxFekhlyMLp3olIrYFfg+c0MqwtgA+DwwFfiapLVEy7+XlVy4CbgKOIuWsjaosqFwtIkZHxOCIGNyjV+82nMLMzNqivTO0PYHtgKl54d49ge8DEyPi5Yh4D7i2Df0MBy6LiLdgqZyzLSVNkjSHtJL9wFb62Rx4JiKeyK8vB3YrvH9D/j4d6NdKX7dGxLs5e+0loMk8syo35+9zgIcj4oWIeBd4GtioDcebmVmNtLegiZTWPCh/bQ6c1sL+izPKJH0EWKWV/scAR+fZ1unULp9sEa3/iUIxy6yyf2sZa5VjPqw6/sM2nM/MzGqovf/o3gXcJOk3EfFSziebAfw232L7F/A1YFbefx5pRvcn0udgldt444BTJV0VEW9JWifP0tYEXsi3+w5kSVZZc/lkjwP9JG0WEU8B3wbubuc1tWRxxhqwkJSxdlutOnd8jJlZ7bRrhhYRjwCnAHdImk0qTH1Js7T7gcmkz74qLgZ2lzQL2BF4M/dzG+l23bR867Ly+dZPgSm5n2Ke2TXAifnhj00L43mHlEc2Nt+m/JD0eVZNRMT7QCVjbRzLZqyZmVmD6BZ5aI3KeWhmZu3nPDQzMyu1bvXggqRDgOOqmidHxFH1GI+ZmdVOtypoEXEZcFm9x2FmZrXnW45mZlYKLmhmZlYK3eqWY6NxHlpjcr6ZWdfkGZqZmZVClyxokjaQdF3eHiRpnzYcM0zSLTUex18k9alln2Zmtny6ZEGLiOcjohJLMwhotaB10Dj2iYjX63FuMzNbWl0KmqTvSJqdc8+ulLSvpCl5aas7Ja2f9zstv39/zhk7PLf3kzRX0iqkpalG5qy0kZKG5v1nSLpP0uZtHNN6ksZJeljSJZKelbRufu/Pkqbn944oHDNP0rp5PI9Kujjvc4ek1Zo5j/PQzMw6QKcXNEkDSetB7pFzz44D7gV2iIjPkNZt/GHhkK2BPUhrQZ4qaYPKGzmu5lTg2rz6/7Wk9RZ3zX2dCvyijUP7GTA+IgYC1wEbF947NCK2AwYDxzaTddYfuDAf/zrw1aZO4jw0M7OOUY+nHPcAxubcMSLiVUlbAddK6kuKmHmmsP9NEfE28LakCaQAzpkt9N8buFxSfyBYssJ/a3YBvpzHdJuk1wrvHSvpy3l7I1Lx+mfV8c9ERGVcbclfMzOzGmqUz9DOBy7IOWjfZencserVk1tbTfnnwISI2BLYlxXMVJM0jBRIumOeUc5ops+m8tTMzKyT1OMf3fHAjZJ+HRH/zJlqvVmSfXZw1f77SzoLWB0YBpzE0kGh1Vlpxb5GtWNck4GvA7+UtBewdqG/13Ju2xbADu3os0XOQzMzq51On6FFxMPAmcDdOSft16Q8tbGSpgOvVB0yG5gAPAD8PCKer3p/AjCg8lAIcA5wlqQZtK9gnw7sJWkuKaT0H6RieRuwkqRHgbPzOMzMrME0dB6apNOAhRHxq044V09gUUR8IGlH4PcRMagjz+k8NDOz9msuD82f8yyxMfAnSR8B3gMOr/N4zMysHRq6oEXEabXus5VMtM/U+nxmZtY5GrqgdQRnopmZlVOjPLZvZma2QrrdDK2ROD6mPhwPY1ZOXX6GJmmUpAtq3OcISQMKr8+QNLyW5zAzs9rq8gWtg4wAFhe0iDg1Iu6s43jMzKwVDV/QJB0k6cH8h9P/LamHpEMkPSHpQWDnwr5jJB1QeL2wsP0jSXPyCv9n57bDJU3NbddL6iVpJ2A/4Nx8zk2L/UraM6/kP0fSpfnv1yor758u6aH83had9CMyMzMavKBJ+jQwEtg5/5HzIuAg0qoeO5MWFB7QfA+L+9kb2B/YPq/HeE5+64aIGJLbHgUOi4j7gJuBE/MK/n8r9LMqMAYYmdedXAk4snCqVyJiW+D3wAnNjMXxMWZmHaChCxqwJ7AdMFXSzPz6+8DEiHg5x8dc24Z+hgOXRcRbkFb4z+1bSpokaQ5wIDCwlX42J62q/0R+fTmwW+H9G/L3Zlfbd3yMmVnHaPSCJuDyPFMaFBGbk9Z9bM4H5GvKK36s0sK+kGZbR+fZ1ums4Mr8LFlx36vtm5l1skYvaHcBB0j6GEBemX8GsLukj0pambSQcMU80owO0udglSy0ccAhknoV+oG0Sv8LuZ8DC/1Ur+Bf8TjQT9Jm+fW3gbuX//LMzKxWGnoWERGPSDoFuCPPuN4HjiLN0u4nJUMXwz4vBm7Kq/jfBryZ+7lN0iBgmqT3gL8APwF+CkwBXs7fK0XsGuBiSccCix8yiYh38tJZYyWtBEwFLlre63N8jJlZ7TT0avtl59X2zczar7nV9hv9lqOZmVmbuKCZmVkpuKCZmVkpuKCZmVkpuKCZmVkpuKCZmVkpNPTfoZWd89A6h/PPzLoHz9DMzKwUunRBk7SBpOvy9iBJ+7ThmGGSbmnh/ZoHhpqZWcfr0gUtIp6PiMrSVIOAVguamZmVU10LmqTvSJqdAzavlLSvpCk5QPNOSevn/U7L798v6UlJh+f2fpLmSloFOAMYmUM5R0oamvefIek+SZsvx/j6SRqfx3iXpI1z+9fyeWdJuie3DSwEkc6W1L+ZPp2HZmbWAer2UIikgcApwE4R8UpeAT+AHSIiJP0b8EPg/+VDtgZ2AFYHZkha/DRFRLwn6VRgcEQcnftfC9g1Ij6QNBz4BfDVdg7zfFJ8zeWSDgXOA0YApwKfj4j5kvrkfb8H/DYirsoFtkdTHUbEaGA0QM++/b2QpplZjdTzKcc9gLER8Qqk0E1JWwHXSupLyjJ7prD/TRHxNvC2pAnAUJZeab9ab+DyPFMKlkTJtMeOwFfy9pUsSbqeDIyR9CeWhHreD5ws6eOkJOwnl+N8Zma2nBrtM7TzgQty4OZ3WTpws3o209rs5ufAhIjYEtiXFQ/vXHLiiO+RZpcbAdMlfTQi/kjKYHsb+IukPWp1PjMza109Z2jjgRsl/Toi/plvOfYG5uf3D67af39JZ5FuOQ4DTmLpROrqUM5iX6OWc4z3Ad8gzc4OBCYBSNo0IqYAUyTtDWwkqTfwdESclz9r2zpfY7Och2ZmVjt1m6FFxMPAmcDdOZDz16TgzrGSpgOvVB0yG5gAPAD8PCKer3p/AjCg8lAI6fbgWZJmsPyF+xhS0vVsUjr1cbn9XElzJM0lFb1ZwNeBuZJmAlsCVyznOc3MbDl0iYBPSacBCyPiV/UeSy054NPMrP0c8GlmZqXWJdZyjIjTat2npENYcguxYnJEHFXrc5mZWcfrEgWtI0TEZcBl9R6HmZnVhm85mplZKXTbGVojcHxM6xz9YmZt1S1maJLmSVq3jfueJumEjh6TmZnVVukLmqQm11Q0M7NyaeiCJulEScfm7d9IGp+395B0laRvVv7AWdIvC8ctlPSf+Q+2dyy0rybpr4XV+pda7b+J8x8uaWp+/3pJvXL7cq+2b2ZmHaOhCxppqald8/ZgYA1JK+e2J4BfkhY5HgQMkTQi77s6MCUitomIe3PbGsD/AldHxMWF1f73iIhtWPYRfkiLDA/J7z8KHJbbK6vtb0NavxGWrLY/KI/1701dkONjzMw6RqMXtOnAdjkK5l3SivaDSQXtdWBiRLwcER8AVwG75eMWAddX9XUTcFlEVJakWma1/ybOv6WkSZLmkNZyHJjbK6vtH86SmJj7gZ9I+hHwiZwMsIyIGB0RgyNicI9evdv+kzAzsxY1dEGLiPdJETKjSGsmTgI+C2wGzGvh0HciYlFV22TgC5LUjiGMAY7Oq/+fTl6x36vtm5k1noYuaNkk4ATgnrz9PWAG8CCwu6R184Mf3wTubqGfU4HXgAvz6/HA1yR9FCCv9l9tTeCFfJvzwEpjZbX9iDgVeJm02v4nyavtk2aDWy/vBZuZWft1hb9DmwScDNwfEW9KegeYFBEvSDqJtMq+gFsj4qZW+joOuFTSORHxQ0mV1f4XkYrkqKr9fwpMIRWtKSyJpzk3P/Qh4C7Savs/Ar4t6X3gH6SE7BY5PsbMrHa6xGr7ZeXV9s3M2s+r7ZuZWam5oJmZWSm4oJmZWSm4oJmZWSm4oJmZWSm4oJmZWSl0hb9DKy3noTXPOWhm1l4NP0OTNFHSMn9vULXP8ZWV8PPrv0jqU8MxNJuRJum+Wp3HzMyWX0MUNCUrMpbjgcUFLSL2iYjXV3xkrYuInTrjPGZm1rK6FTRJ/SQ9LukKYC5p2aj7JT0kaaykNZo45vc5euVhSafntmOBDYAJkibktsUJ1ZJ+kLPL5ko6vnDuRyVdnPu6Q9Jqlf4kPZIzza4pnH5Ani0+Xcloy/svzN+HSbpH0q35ui5awSJtZmbtUO9/cPsDvwN2J2WNDY+IbYFpwA+a2P/kvNzJ1qSFibfOiwE/D3w2Ij5b3FnSdsAhwPbADsDhkj5TOPeFETGQFEXz1dx+EvCZiNiatBByxRbA54GhwM/ygsXVhgLHAAOATYGvVO/gPDQzs45R74L2bEQ8QCo2A4DJkmYCBwOfaGL/r0t6iLSQ8MB8TEt2AW6MiDcjYiFwA0sCQ5+JiJl5ezrQL2/PBq6SdBDwQaGvWyPi3Zyf9hKwfhPnezAins7RNVfn8y/FeWhmZh2j3k85vpm/CxgXEd9sbkdJm5BiZIZExGuSxpDzyZbTu4XtRcBqefuLpKDQfYGTJW3VzP5N/eyqV3r2ys9mZp2k3jO0igeAnSVtBiBpdUmfqtpnLVIBXCBpfWDvwntvsCTapWgSMEJSL0mrA1/ObU3Kn3ltFBETSHEwvYFlPstrwVBJm+R+RgL3tuNYMzNbAfWeoQEQES9LGgVcLalnbj4FeKKwzyxJM4DHgOdICdQVo4HbJD1f/BwtIh7KM7kHc9MlETFDUr9mhtID+IOk3qRZ43kR8Xo7Qq6nAheQErUnADe2tLPz0MzMasd5aDUiaRhwQkR8qa3HOA/NzKz9nIdmZmal1hC3HMsgIiYCE+s8DDOzbsszNDMzKwUXNDMzKwUXNDMzKwUXNDMzKwU/FFJH3TEPzTlnZtZRGm6GJmkDSdfl7UGS9mnDMcMk3dLO8/ST9K1a7WdmZvXVUAVN0koR8XxEHJCbBgGtFrTl1A9oS6Fq635mZlZHNSloeRbzmKQxkp6QdJWk4ZImS3pS0tD8db+kGZLuk7R5PnaUpJsljQfuyn3NlbQKcAYwUtJMSSOb66MN49s99zEzH7smcDawa277fj7vpJzH9pCkSnBn9X6jJF1Q6PuWPEPska9/rqQ5kr7fzFgcH2Nm1gFq+RnaZsDXgENJaxp+ixSfsh/wE+A7wK4R8YGk4cAvWJJBti2wdUS8WllnMSLek3QqMDgijgaQtFYLfbTkBOCoiJisFBz6Din3bPFSVZJ6AZ+LiHck9SfFvwxuYr9RzZxjELBhRGyZ9+vT1E4RMZq09iQ9+/b3umNmZjVSy4L2TETMAZD0MHBXRISkOaTbdr2By3OxCKAYkDkuIl5twzla6qMlk4FfS7oKuCEi/t7EgsMrAxdIGkSKh6le7b81TwOflHQ+cCtwRzuPNzOzFVDLz9CKeWEfFl5/SCqcPwcm5BnMviydZfYmbdNSH82KiLOBfyNlnk2WtEUTu30feBHYhjQzW6WZ7j5g6Z/bqvkcr+VjJ5KSri9py9jMzKw2OvOx/d7A/Lw9qo3HVOecLU8fSNo0zx7nSBoCbEGKoKnu++8R8aGkg0lRMk2NYR7w7znzbENgaD7HusB7EXG9pMeBP7Q2LsfHmJnVTmc+5XgOcFbONGtrIZ0ADKg8FLKcfQAcnx/WmA28D/wVmA0skjQrP8DxO+BgSbNIBa8ya6zebzLwDPAIcB7wUN5vQ2CipJmkYvbjdozPzMxWkPPQ6sh5aGZm7ec8NDMzK7VSLX0l6RDguKrmyRFxVD3GY2ZmnadUBS0iLgMuq/c4zMys8/mWo5mZlYILmpmZlUKpbjl2NV0pPsaxL2bW6DxDMzOzUui0giZphKQBhddn5AWGu4Tq8ZuZWWPpzBnaCGBxQYiIUyPizk48/4paavxFknzr1syszjVS4r0AAAodSURBVJa7oOX8sEclXSzpYUl3SFpN0uGSpualoq6X1Ctni+0HnJuXsdo0Z4cdIOkLksYW+l2cPi1pr5x/9pCksTn6pbnxDMkZabMkPShpTUmrSros55PNkPTZvG+TmWZ5e6GkM3M/D0hav5nxT5T0X5KmASdLekbSyrmPtYqvq8bpPDQzsw6wojO0/sCFETEQeJ2UTXZDRAyJiG2AR4HDIuI+4GbgxIgYFBF/K/RxJ7C9pNXz65HANXmx31OA4RGxLTAN+EFTg1AKA70WOC6fdzjwNnAUEBGxFfBNUvRMayv0rw48kPu5Bzi8hfGvEhGDI+J00ir7lScnvpF/Du9Xdx4Ro/Mxg3v06t3KUMzMrK1WtKA9ExEz8/Z0Uu7ZlkrJz3OAA4GBLXUQER8AtwH75lt3XwRuAnYg3eKbnBf8PRj4RDPdbA68EBFTc5//yv3uQl71PiIeA56l9Zyz94Bbqq6pOdcWti8BDsnbh+A/8DYz61Qr+tlPMQNtESlvbAwwIiJm5XTnYW3o5xrgaOBVYFpEvKGUwDkuIr65gmNsSpOZZtn7sWTF5kW0/DNanOOW07D75VuXPSJibq0Ga2ZmreuIhxnWBF7Inx8dyJL8supcsaK7gUuBw0nFDeAB4EJJm0XEU/mW5IYR8UQTxz8O9JU0JCKmSlqTdMtxUh7DeEmfAjbO+65FE5lmrWhp/BVXAH8kBZG2ynloZma10xFPOf4UmELKDXus0H4NcGJ+OGPT4gERsYh0m2/v/J2IeJkU4nl1zjG7n5RTtoyIeI/02dv5Oc9sHGnW9TvgI/n257XAqIh4l+YzzVrS7PgLrgLWBq5uQ39mZlZDzkOrIUkHAPtHxLfbsr/z0MzM2q+5PDT//VSNSDqfNMPcp95jMTPrjrpcQZN0I7BJVfOPIuL2eoynIiKOqef5zcy6uy5X0CLiy/Ueg5mZNR4vTmxmZqXggmZmZqXQ5W45lkk989Ccb2ZmZeMZmpmZlYILWgskrS7p1rzy/lxJIyVtJ+luSdMl3S6pr6SVcsLAsHzcWZLOrPPwzcy6Fd9ybNkXgOcj4osAknoDfyX98fTLkkYCZ0bEoXndyuskHZOP275egzYz645c0Fo2B/hPSb8kLcn1GrAlMC6tnUwP4AWAiHhY0pV5vx3zclzLkHQEcARAj7XW6/ALMDPrLlzQWhART0jalrT6x38A44GHI2LHZg7ZipQL97EW+hwNjAbo2be/1x0zM6sRf4bWAkkbAG9FxB+Ac0m3EdeTtGN+f2VJA/P2V4B1gN1IiyT3qdOwzcy6Jc/QWrYVcK6kD4H3gSNJWWrn5c/TVgL+S9KLwNnAnhHxnKQLgN+SQkmb79zxMWZmNeOC1oK8PmRTa0Tu1kTb4iTsiDivwwZlZmZN8i1HMzMrBRc0MzMrBRc0MzMrBRc0MzMrBRc0MzMrBRc0MzMrBT+2X0f1io9xdIyZlZFnaGZmVgouaM1Q4p+PmVkX4X+wCyT1k/S4pCuAucD/SJom6WFJpxf2GyLpvpyT9qCkNSX1kHRuzkWbLem79bsSM7Pux5+hLas/cHBEPCBpnYh4VVIP4C5JWwOPAdcCIyNiqqS1gLeBw4AFETFEUk9gsqQ7IuKZYueOjzEz6xguaMt6NiIeyNtfzwVoJaAvMAAI4IWImAoQEf8CkLQXsLWkA/KxvUnFcamC5vgYM7OO4YK2rDcBJG0CnAAMiYjXJI0BVm3hOAHH5AWNzcysk/kztOatRSpuCyStD+yd2x8H+koaApA/P1uJtCr/kZJWzu2fkrR6HcZtZtYteYbWjIiYJWkG6TOz54DJuf09SSNJIZ6rkT4/Gw5cAvQDHpIk4GVgREvncB6amVntKMIf49TL4MGDY9q0afUehplZlyJpekQMrm73LUczMysFFzQzMysF33KsI0lvkB4yKat1gVfqPYgO5Ovr+sp+jWW9vk9ExDJ/yOuHQurr8abuA5eFpGm+vq6r7NcH5b/Gsl9fNd9yNDOzUnBBMzOzUnBBq6/R9R5AB/P1dW1lvz4o/zWW/fqW4odCzMysFDxDMzOzUnBBMzOzUnBBqwNJX8hBok9JOqne42kPSZdKeknS3ELbOpLGSXoyf187t0vSefk6Z0vatnDMwXn/JyUdXI9rqSZpI0kTJD2SQ12Py+2luD4ASavmUNpZxeBaSZtImpKv5VpJq+T2nvn1U/n9foW+fpzbH5f0+fpcUdNy4O4MSbfk16W5PknzJM2RNFPStNxWmt/RFRIR/urEL6AH8Dfgk8AqwCxgQL3H1Y7x7wZsC8wttJ0DnJS3TwJ+mbf3Af5KitbZAZiS29cBns7f187bazfAtfUFts3bawJPkDLwSnF9eWwC1sjbKwNT8tj/BHwjt18EHJm3/x24KG9/A7g2bw/Iv7s9gU3y73SPel9f4Tp/APwRuCW/Ls31AfOAdavaSvM7uiJfnqF1vqHAUxHxdES8B1wD7F/nMbVZRNwDvFrVvD9wed6+nCUpA/sDV0TyANBHUl/g88C4iHg1Il4DxgFf6PjRtywiXoiIh/L2G8CjwIaU5PoA8lgX5pcr568A9gCuy+3V11i59uuAPXOaxP7ANRHxbqRU9qdIv9t1J+njwBdJCRjk8Zbm+ppRmt/RFeGC1vk2JMXRVPw9t3Vl60fEC3n7H8D6ebu5a234n0G+9fQZ0gymVNeXb8fNBF4i/UP2N+D1iPgg71Ic7+Jrye8vAD5KY1/jfwE/BD7Mrz9Kua4vgDskTZd0RG4r1e/o8vLSV1ZTERGSuvTfgkhaA7geOD4i/pX+hz0pw/VFxCJgkKQ+wI3AFnUeUs1I+hLwUkRMlzSs3uPpILtExHxJHwPGSXqs+GYZfkeXl2donW8+sFHh9cdzW1f2Yr6NQf7+Um5v7lob9meglDh+PXBVRNyQm0tzfUUR8TowAdiRdCuq8j+4xfEuvpb8fm/gnzTuNe4M7CdpHul2/h7AbynP9RER8/P3l0j/QzKUkv6OtpcLWuebCvTPT12tQvog+uY6j2lF3QxUnpI6GLip0P6d/KTVDsCCfFvkdmAvSWvnp7H2ym11lT87+R/g0Yj4deGtUlwfgKT18swMpcT1z5E+K5wAHJB3q77GyrUfAIyP9FTBzcA38lOCmwD9gQc75yqaFxE/joiPR0Q/0n9b4yPiQEpyfZJWl7RmZZv0uzWXEv2OrpB6P5XSHb9ITx49Qfrs4uR6j6edY78aeAF4n3Tf/TDSZw53AU8CdwLr5H0FXJivcw4wuNDPoaQP2p8CDqn3deUx7UL6fGI2MDN/7VOW68vj2hqYka9xLnBqbv8k6R/sp4CxQM/cvmp+/VR+/5OFvk7O1/44sHe9r62Jax3GkqccS3F9+Tpm5a+HK/9+lOl3dEW+vPSVmZmVgm85mplZKbigmZlZKbigmZlZKbigmZlZKbigmZlZKbigmZlZKbigmZlZKfx/xeuu2gKhL/cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# question 7: plot feature importance\n",
    "feature_imp = xgb.train(param, train_initial, num_boost_round = df_cv.shape[0])\n",
    "imp_dict = feature_imp.get_score()\n",
    "df_imp = pd.DataFrame.from_dict(imp_dict, orient = 'index')\n",
    "df_imp.columns = ['importance']\n",
    "df_imp = df_imp['importance'].sort_values()\n",
    "df_imp.plot.barh()\n",
    "plt.title('importance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7005969160764121"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# question 8: analyze auc score on Test Data,\n",
    "test_initial = xgb.DMatrix(x_test, y_test)\n",
    "predict_result = feature_imp.predict(test_initial)\n",
    "roc_auc_score(y_test, predict_result.round())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
